package common.main.sougouWeixin;

import common.http.HtmlInfo;
import common.http.SimpleHttp;
import common.system.FileOperation;
import common.utils.DomTree;
import common.utils.MD5Util;
import org.apache.log4j.Logger;
import org.w3c.dom.DocumentFragment;
import org.w3c.dom.NodeList;

import java.text.SimpleDateFormat;
import java.util.ArrayList;
import java.util.Date;
import java.util.List;

public class SougouWeixinMain {
    static Logger logger = Logger.getLogger(SougouWeixinMain.class);

    static String ua = "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/68.0.3440.106 Safari/537.36";
    static SimpleHttp http = new SimpleHttp();
    static String  modelUrl = "http://weixin.sogou.com/weixin?type=2&query=<searchKeyword>&ie=utf8&s_from=input&_sug_=n&_sug_type_=1";
    static String allMd5;

    public static void main(String[] args) throws InterruptedException {
        SougouWeixinConfig.init("config/sougouWeixinConfig.txt");
        String nextUrl = null;
        allMd5 = SougouWeixinSaveDataToSql.getAllMd5();

        if(SougouWeixinConfig.entranceUrl == null){
            nextUrl = modelUrl.replace("<searchKeyword>","知识图谱");
        }else{
            nextUrl = SougouWeixinConfig.entranceUrl;//html.setOrignUrl();
        }
        String referUrl = "http://weixin.sogou.com/weixin?type=2&query=%E7%9F%A5%E8%AF%86%E5%9B%BE%E8%B0%B1&ie=utf8&s_from=input&_sug_=n&_sug_type_=1";
        while (nextUrl != null){
            HtmlInfo html = new HtmlInfo();
            String loginCookie = SougouWeixinConfig.cookie;
            html.setCookie(loginCookie);
            html.setReferUrl(referUrl);
            html.setUa(ua);
            html.setEncode("utf-8");
            logger.info("当前url为：" + nextUrl);
            FileOperation.appendWrite("当前url为：" + nextUrl + "\n","config/sougouWeixinUrl.txt");
            html.setOrignUrl(nextUrl);
            http.simpleGet(html);
            //如果访问频繁，则退出程序。
            if(html.getContent() != null && html.getContent().contains("您的访问过于频繁")){
                System.exit(-1);
            }
            List<SougouWeixinData> list = new ArrayList<>();

            DocumentFragment listNode = DomTree.getNode(html.getContent(),html.getEncode());
            referUrl = nextUrl;
            nextUrl = extractPaperInfo(list,listNode);

            if(list.size() > 0){
                extractOther(list,listNode);
                SougouWeixinSaveDataToSql.saveDataToSql(list);
            }

            logger.info("等待15s 获取下一页内容");
            Thread.sleep(15 * 1000);
        }
    }

    public static String extractPaperInfo(List<SougouWeixinData> list,DocumentFragment node){
        logger.info("获取列表信息");
        NodeList paperInfoNodeList = DomTree.commonList(SougouWeixinXpath.paperInfo,node);
        if(paperInfoNodeList != null){
            for(int i = 0;i < paperInfoNodeList.getLength();i++){
                SougouWeixinData data = new SougouWeixinData();
                data.setPaperInfo(paperInfoNodeList.item(i).getTextContent());
                list.add(data);
            }
        }
        String nextUrl = null;
        NodeList nextUrlNodeList = DomTree.commonList(SougouWeixinXpath.nextPageUrl,node);
        if(nextUrlNodeList != null && nextUrlNodeList.getLength() > 0){
            nextUrl = "http://weixin.sogou.com/weixin" + nextUrlNodeList.item(0).getTextContent();
        }
        return nextUrl;
    }

    public static void extractOther(List<SougouWeixinData> list,DocumentFragment node) throws InterruptedException {
        logger.info("获取文章的列表信息");
        //抽取摘要
        NodeList briefNodeList = DomTree.commonList(SougouWeixinXpath.brief,node);
        if(briefNodeList != null){
            for(int i = 0;i < briefNodeList.getLength();i++){
                String brief = briefNodeList.item(i).getTextContent();
                for(int j = 0;j < list.size();j++){
                    SougouWeixinData data = list.get(j);
                    if(data.getPaperInfo().contains(brief)){
                        data.setBrief(brief);
                        break;
                    }
                }
            }
        }
        //抽取标题
        NodeList titleNodeList = DomTree.commonList(SougouWeixinXpath.title,node);
        if(titleNodeList != null){
            for(int i = 0;i < titleNodeList.getLength();i++){
                String title = titleNodeList.item(i).getTextContent();
                String url = titleNodeList.item(i).getAttributes().getNamedItem("href").getTextContent();
                for(int j = list.size() - 1;j >= 0;j--){
                    SougouWeixinData data = list.get(j);
                    if(data.getPaperInfo().contains(title)){
                        data.setTitle(title);
                        String md5 = MD5Util.MD5(data.getTitle() + data.getBrief());
                        if(!allMd5.contains(md5)){
                            HtmlInfo html = new HtmlInfo();
                            String loginCookie = SougouWeixinConfig.cookie;
                            html.setCookie(loginCookie);
                            html.setUa(ua);
                            html.setEncode("utf-8");

                            data.setMd5(md5);
                            data.setUrl(url);
                            logger.info("15s后获取《" + data.getTitle() + "》的详细信息");
                            Thread.sleep(15 * 1000);
                            html.setOrignUrl(url);
                            http.simpleGet(html);
                            DocumentFragment contentNode = DomTree.getNode(html.getContent(),html.getEncode());
                            NodeList contentNodeList = DomTree.commonList(SougouWeixinXpath.content,contentNode);
                            if(contentNodeList != null ){
                                String content = "";
                                for(int a = 0;a < contentNodeList.getLength();a++){
                                    content = content + contentNodeList.item(a).getTextContent() + "\n";
                                }
                                data.setContent(content);
                            }
                            NodeList imgUrlNodeList = DomTree.commonList(SougouWeixinXpath.imgUrl,contentNode);
                            if(imgUrlNodeList != null){
                                String imgUrl = "";
                                for(int k = 0;k < imgUrlNodeList.getLength();k++){
                                    imgUrl = imgUrl + imgUrlNodeList.item(k).getTextContent() + "<;>";
                                }
                                data.setImgUrl(imgUrl);
                            }
                            data.setSearchKeword("知识图谱");
                            allMd5 = allMd5 + md5 + ",";
                        }else{
                            logger.info("文章《" + data.getTitle() + "》已经存在数据库中，跳过采集");
                            list.remove(j);
                        }
                        break;
                    }
                }
            }
        }


        //抽取作者
        NodeList authorNodeList = DomTree.commonList(SougouWeixinXpath.author,node);
        if(authorNodeList != null){
            for(int i = 0;i < authorNodeList.getLength();i++){
                String author = authorNodeList.item(i).getTextContent();
                for(int j = 0;j < list.size();j++){
                    SougouWeixinData data = list.get(j);
                    if(data.getPaperInfo().contains(author)){
                        data.setAuthor(author);
                        break;
                    }
                }
            }
        }
        //抽取发表时间
        NodeList pubTimeNodeList = DomTree.commonList(SougouWeixinXpath.pubTime,node);
        if(pubTimeNodeList != null){
            SimpleDateFormat sdf = new SimpleDateFormat("yyyy-MM-dd HH:mm:ss");
            for(int i = 0;i < pubTimeNodeList.getLength();i++){
                String pubTime = pubTimeNodeList.item(i).getTextContent();
                for(int j = 0;j < list.size();j++){
                    SougouWeixinData data = list.get(j);
                    if(data.getPaperInfo().contains(pubTime)){
                        long pubTimeLong = Long.valueOf(pubTime);
                        Date date = new Date(pubTimeLong * 1000);
                        String pubTimeStr = sdf.format(date);
                        data.setPubTime(pubTimeStr);
                        break;
                    }
                }
            }
        }
    }
}
